# -*- coding: utf-8 -*-
"""
Houzz URL Collector (v1.3 - Per-Loop Latency Re-Test Mode)
- Purpose: Traverses category list pages to extract detailed profile URLs.
- Storage: Saves URLs to a MySQL database.
- Proxy Support: Re-tests latency for all nodes before EACH task, ensuring
  the absolute best node is always chosen for the job.
- Robustness: Failed tasks are re-queued to be retried later with a fresh
  set of latency-tested proxies.
"""
import sys
import json
import logging
import os
import time
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from lxml import etree
from tqdm import tqdm
import urllib.parse
from concurrent.futures import ThreadPoolExecutor, as_completed
import pymysql
from pymysql.cursors import DictCursor
from collections import deque

# --- 1. Configuration ---
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler("houzz_url_collector.log", mode='a', encoding='utf-8'),
        logging.StreamHandler()
    ]
)

# !!! IMPORTANT: CONFIGURE YOUR DATABASE CONNECTION HERE !!!
DB_CONFIG = {

}

CHECKPOINT_FILE = "collector_checkpoint.json"


# --- 2. Clash Proxy Management Module ---
class ClashAPI:
    """A controller to interact with the Clash API for proxy management."""

    def __init__(self, api_host, secret=None):
        self.base_url = api_host.rstrip('/')
        self.headers = {'Content-Type': 'application/json'}
        if secret:
            self.headers['Authorization'] = f'Bearer {secret}'
        try:
            response = requests.get(f'{self.base_url}/version', headers=self.headers, timeout=10, proxies=None)
            response.raise_for_status()
            logging.info(f"✅ Clash API connection successful! Version: {response.json().get('version')}")
        except requests.exceptions.RequestException as e:
            logging.error(f"❌ FATAL: Could not connect to Clash API at {api_host}. Error: {e}")
            sys.exit(1)

    def get_full_proxies_info(self) -> dict:
        """Retrieves information about all available proxies and groups."""
        url = f'{self.base_url}/proxies'
        try:
            response = requests.get(url, headers=self.headers, timeout=10, proxies=None)
            response.raise_for_status()
            return response.json().get('proxies', {})
        except requests.exceptions.RequestException as e:
            logging.error(f"❌ Failed to get proxy info from Clash API: {e}")
            return {}

    def switch_node(self, group_name: str, node_name: str) -> bool:
        """Switches the active node for a specified policy group."""
        logging.info(f"🔄 Attempting to switch Clash node to: [{node_name}]")
        encoded_group_name = urllib.parse.quote(group_name)
        url = f'{self.base_url}/proxies/{encoded_group_name}'
        payload = json.dumps({'name': node_name})
        try:
            response = requests.put(url, headers=self.headers, data=payload, timeout=10, proxies=None)
            response.raise_for_status()
            logging.info(f"✅ Node switch successful!")
            return True
        except requests.exceptions.RequestException as e:
            error_text = e.response.text if e.response else str(e)
            logging.error(f"❌ Failed to switch to node [{node_name}]: {error_text}")
            return False

    def check_node_latency(self, node_name: str, timeout_ms: int) -> int:
        """Tests the latency of a specific node."""
        try:
            encoded_node_name = urllib.parse.quote(node_name)
            test_url = "http://www.gstatic.com/generate_204"
            url = f'{self.base_url}/proxies/{encoded_node_name}/delay?timeout={timeout_ms}&url={test_url}'
            response = requests.get(url, headers=self.headers, timeout=(timeout_ms / 1000) + 2, proxies=None)
            response.raise_for_status()
            return response.json().get('delay', 9999)
        except requests.exceptions.RequestException:
            return 9999


def get_valid_clash_nodes(clash_api_controller: ClashAPI, policy_group: str) -> list[str]:
    """Filters and returns a list of valid, usable proxy nodes from a policy group."""
    all_proxies = clash_api_controller.get_full_proxies_info()
    if not all_proxies: return []
    if policy_group not in all_proxies or 'all' not in all_proxies[policy_group]:
        logging.error(f"❌ FATAL: Policy group '{policy_group}' not found in Clash config.")
        return []
    group_members = all_proxies[policy_group]['all']
    valid_node_types = ['Vmess', 'Trojan', 'Shadowsocks', 'Socks', 'Http', 'Ss']
    invalid_keywords = ['流量', '到期', '官网', '客服', '剩余', '时间', '发布页']
    available_nodes = [
        name for name in group_members
        if name in all_proxies
           and all_proxies[name].get('type') in valid_node_types
           and not any(keyword in name for keyword in invalid_keywords)
    ]
    if not available_nodes:
        logging.warning("   No valid, switchable proxy nodes found in the policy group.")
    return available_nodes


def get_and_sort_nodes_by_latency(clash_api_controller: ClashAPI, policy_group: str, timeout_ms: int) -> list[str]:
    """Tests latency for all valid nodes in parallel and returns them sorted from best to worst."""
    initial_nodes = get_valid_clash_nodes(clash_api_controller, policy_group)
    if not initial_nodes: return []
    node_latencies = {}
    with ThreadPoolExecutor(max_workers=20, thread_name_prefix='LatencyTester') as executor:
        future_to_node = {executor.submit(clash_api_controller.check_node_latency, node, timeout_ms): node for node in
                          initial_nodes}
        # 使用 no-leave 的 tqdm，这样每次测试完进度条会消失，保持界面整洁
        for future in tqdm(as_completed(future_to_node), total=len(initial_nodes),
                           desc="Testing node latencies for current task", leave=False):
            node_name = future_to_node[future]
            try:
                latency = future.result()
                if latency < 9999:
                    node_latencies[node_name] = latency
            except Exception:
                pass
    if not node_latencies:
        logging.error("❌ All nodes failed the latency test. No proxies are available for this cycle.")
        return []
    sorted_nodes = sorted(node_latencies.items(), key=lambda item: item[1])
    best_node, best_latency = sorted_nodes[0]
    logging.info(
        f"✅ Latency test complete. {len(sorted_nodes)} nodes available. Best node: [{best_node}] with {best_latency}ms.")
    return [node for node, latency in sorted_nodes]


# --- 3. Database Operations ---
def get_db_connection():
    """Establishes and returns a connection to the MySQL database."""
    try:
        connection = pymysql.connect(**DB_CONFIG)
        return connection
    except pymysql.MySQLError as e:
        logging.error(f"❌ FATAL: Database connection failed: {e}")
        sys.exit(1)


def save_urls_to_db(category_url: str, urls: list):
    """Saves a batch of profile URLs to the database."""
    if not urls: return 0
    conn = get_db_connection()
    try:
        with conn.cursor() as cursor:
            sql = "INSERT IGNORE INTO `urls_to_scrape` (`category_url`, `profile_url`) VALUES (%s, %s)"
            data_to_insert = [(category_url, url) for url in urls]
            inserted_rows = cursor.executemany(sql, data_to_insert)
            conn.commit()
            logging.info(f"  - Database: Inserted {inserted_rows} new URLs (out of {len(urls)} found).")
            return inserted_rows
    except pymysql.MySQLError as e:
        logging.error(f"❌ Database insert operation failed: {e}")
        conn.rollback()
        return 0
    finally:
        conn.close()


# --- 4. Checkpoint Management ---
def save_checkpoint(filename: str, url: str, page_num: int):
    """Saves the last successfully processed page number for a given category URL."""
    data = {}
    if os.path.exists(filename):
        try:
            with open(filename, 'r', encoding='utf-8') as f:
                content = f.read()
                if content: data = json.loads(content)
        except (IOError, json.JSONDecodeError):
            data = {}
    data[url] = page_num
    with open(filename, 'w', encoding='utf-8') as f:
        json.dump(data, f, indent=4)


def load_checkpoint(filename: str, url: str) -> int:
    """Loads the last successfully processed page number for a given category URL."""
    if not os.path.exists(filename): return 0
    try:
        with open(filename, 'r', encoding='utf-8') as f:
            content = f.read()
            if not content: return 0
            data = json.loads(content)
            return data.get(url, 0)
    except (IOError, json.JSONDecodeError):
        return 0


# --- 5. Core Networking Function ---
def fetch_url(session: requests.Session, url: str, params: dict = None, retries: int = 3, timeout: int = 30):
    """
    A robust function to fetch a URL with configurable retries and exponential backoff.
    Special handling for 404 errors, returning 'not_found'.
    """
    for attempt in range(retries):
        try:
            response = session.get(url, params=params, timeout=timeout)
            response.raise_for_status()  # Raises HTTPError for 4xx/5xx responses
            return response
        except requests.exceptions.HTTPError as e:
            if e.response.status_code == 404:
                logging.warning(
                    f"Request for {url} with params {params} returned 404 Not Found. Assuming end of category.")
                return 'not_found'
            logging.warning(f"Request failed for {url} (Attempt {attempt + 1}/{retries}): {e}")
            if attempt < retries - 1: time.sleep(2 ** attempt)
        except requests.exceptions.RequestException as e:
            logging.warning(f"Request failed for {url} (Attempt {attempt + 1}/{retries}): {e}")
            if attempt < retries - 1: time.sleep(2 ** attempt)
    return None


# --- 6. Main Task Executor ---
def run_collection_task(base_url: str, config: dict, session: requests.Session) -> str:
    """
    Executes the URL collection process for a single category.
    Returns 'success' if completed, 'failure' otherwise.
    """
    # 使用正则表达式提取类目名称
    import re
    pattern = r'/professionals/([^/]+)(?:/|$)'
    match = re.search(pattern, base_url)
    if match:
        category_name = match.group(1)
    else:
        # 备用方法
        category_name = base_url.split('/')[4] if '/' in base_url else base_url
    try:
        last_completed_page = load_checkpoint(CHECKPOINT_FILE, base_url)
        start_page = last_completed_page + 1 if last_completed_page > 0 else config['START_PAGE']

        if start_page > config['END_PAGE']:
            logging.info(f"Category '{category_name}' has already been completed according to checkpoint. Skipping.")
            return 'success'

        logging.info(f"Processing category '{category_name}' from page {start_page} to {config['END_PAGE']}.")

        page_range = range(start_page, config['END_PAGE'] + 1)
        progress_bar = tqdm(page_range, desc=f"Collecting '{category_name}'")

        for page_num in progress_bar:
            list_params = {"fi": str((page_num - 1) * 15)}
            list_response = fetch_url(session, base_url, list_params, retries=config['RETRY_COUNT'],
                                      timeout=config['REQUEST_TIMEOUT'])

            if list_response == 'not_found':
                logging.info(f"Page {page_num} for '{category_name}' does not exist (404). Assuming end of category.")
                save_checkpoint(CHECKPOINT_FILE, base_url, config['END_PAGE'])
                return 'success'

            if not list_response:
                logging.error(f"Failed to fetch page {page_num} for category '{category_name}' after multiple retries.")
                return 'failure'

            html_tree = etree.HTML(list_response.text)
            profile_urls = html_tree.xpath('//ul[contains(@class, "hz-pro-search-results")]//li/a/@href')

            if not profile_urls and page_num > 1:
                logging.info(
                    f"No profile URLs found on page {page_num} for '{category_name}'. Assuming end of category.")
                save_checkpoint(CHECKPOINT_FILE, base_url, config['END_PAGE'])
                return 'success'

            save_urls_to_db(base_url, profile_urls)
            save_checkpoint(CHECKPOINT_FILE, base_url, page_num)

        logging.info(f"Successfully completed all configured pages for category '{category_name}'.")
        return 'success'

    except Exception as e:
        logging.error(f"An unexpected error occurred while processing '{category_name}': {e}", exc_info=True)
        return 'failure'


# --- 7. Main Program Entry Point ---
def main():
    """Main function to configure and run the URL collection tasks."""
    initial_tasks = [
        {
            "url": "https://www.houzz.com/professionals/general-contractor/project-type-green-building-probr1-bo~t_11786~sv_29328",
            "end_page": 762},
        {
            "url": "https://www.houzz.com/professionals/general-contractor/project-type-home-inspection-probr1-bo~t_11786~sv_29344",
            "end_page": 394},
        {
            "url": "https://www.houzz.com/professionals/general-contractor/project-type-exterior-door-installation-probr1-bo~t_11786~sv_29281",
            "end_page": 1586},
        {
            "url": "https://www.houzz.com/professionals/general-contractor/project-type-demolition-probr1-bo~t_11786~sv_29245",
            "end_page": 1785},
        {
            "url": "https://www.houzz.com/professionals/general-contractor/project-type-drywall-installation-probr1-bo~t_11786~sv_29265",
            "end_page": 1921},
        {
            "url": "https://www.houzz.com/professionals/hznb/probr2-bo~t_11786~s_2103~sv_29156",
            "end_page": 40},
        {
            "url": "https://www.houzz.com/professionals/hznb/probr2-bo~t_11786~s_2105~sv_29156",
            "end_page": 37},
        {
            "url": "https://www.houzz.com/professionals/hznb/probr2-bo~t_11786~s_2107~sv_29156",
            "end_page": 41},
        {
            "url": "https://www.houzz.com/professionals/hznb/probr2-bo~t_11786~s_2115~sv_29156",
            "end_page": 7},
        {
            "url": "https://www.houzz.com/professionals/hznb/probr2-bo~t_11786~s_2114~sv_29156",
            "end_page": 14},
        {
            "url": "https://www.houzz.com/professionals/hznb/probr2-bo~t_11786~s_2112~sv_29156",
            "end_page": 33},
        {
            "url": "https://www.houzz.com/professionals/general-contractor/project-type-bathroom-remodeling-probr1-bo~t_11786~sv_29156",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/general-contractor/project-type-custom-homes-probr1-bo~t_11786~sv_29222",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/general-contractor/project-type-new-home-construction-probr1-bo~t_11786~sv_29402",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/general-contractor/project-type-kitchen-remodeling-probr1-bo~t_11786~sv_29368",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/architect/probr0-bo~t_11784",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/design-build/probr0-bo~t_11793",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/home-builders/probr0-bo~t_11823",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/interior-designer/probr0-bo~t_11785",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/kitchen-and-bath-remodelers/probr0-bo~t_11825",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/landscape-architect/probr0-bo~t_11788",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/landscape-contractors/probr0-bo~t_11812",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/adu-contractors/probr0-bo~t_34256",
            "end_page": 1563},
        {
            "url": "https://www.houzz.com/professionals/home-remodeling/probr0-bo~t_34257",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/home-additions-and-extensions/probr0-bo~t_34259",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/kitchen-and-bath/project-type-kitchen-design-probr1-bo~t_11790~sv_29367",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/kitchen-and-bath/project-type-bathroom-design-probr1-bo~t_11790~sv_29155",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/kitchen-and-bath/project-type-design-consultation-probr1-bo~t_11790~sv_30902",
            "end_page": 300},
        {
            "url": "https://www.houzz.com/professionals/kitchen-and-bath/project-type-outdoor-kitchen-design-probr1-bo~t_11790~sv_30765",
            "end_page": 300},

    ]
    # initial_tasks = [
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-grout-repair-probr1-bo~t_11786~sv_30907",
    #         "end_page": 280},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-modular-home-additions-probr1-bo~t_11786~sv_30769",
    #         "end_page": 335},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-modular-home-construction-probr1-bo~t_11786~sv_30768",
    #         "end_page": 334},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-dock-design-and-construction-probr1-bo~t_11786~sv_30753",
    #         "end_page": 315},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-range-hood-installation-probr1-bo~t_11786~sv_30749",
    #         "end_page": 502},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-tiny-house-construction-probr1-bo~t_11786~sv_30721",
    #         "end_page": 593},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-attic-conversion-probr1-bo~t_11786~sv_29734",
    #         "end_page": 1122},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-project-management-probr1-bo~t_11786~sv_29723",
    #         "end_page": 1618},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-home-gym-design-and-construction-probr1-bo~t_11786~sv_29721",
    #         "end_page": 539},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-chimney-construction-probr1-bo~t_11786~sv_29695",
    #         "end_page": 343},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-pool-house-design-and-construction-probr1-bo~t_11786~sv_29686",
    #         "end_page": 684},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-home-restoration-probr1-bo~t_11786~sv_29675",
    #         "end_page": 1369},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-barn-design-and-construction-probr1-bo~t_11786~sv_29578",
    #         "end_page": 632},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-wheelchair-ramp-installation-probr1-bo~t_11786~sv_29554",
    #         "end_page": 442},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-waterproofing-probr1-bo~t_11786~sv_29549",
    #         "end_page": 500},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-trim-work-probr1-bo~t_11786~sv_29522",
    #         "end_page": 1319},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-subfloor-installation-probr1-bo~t_11786~sv_29495",
    #         "end_page": 889},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-stucco-repair-probr1-bo~t_11786~sv_29492",
    #         "end_page": 406},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-soffit-repair-probr1-bo~t_11786~sv_29475",
    #         "end_page": 648},
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-guesthouse-design-and-construction-probr1-bo~t_11786~sv_30778",
    #         "end_page": 736
    #     },
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-soffit-installation-probr1-bo~t_11786~sv_29474",
    #         "end_page": 691
    #     },
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-sliding-door-installation-probr1-bo~t_11786~sv_29468",
    #         "end_page": 1108
    #     },
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-skylight-installation-probr1-bo~t_11786~sv_29466",
    #         "end_page": 600
    #     },
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-site-preparation-probr1-bo~t_11786~sv_29465",
    #         "end_page": 530
    #     }, {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-porch-design-and-construction-probr1-bo~t_11786~sv_29430",
    #         "end_page": 1240
    #     }, {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-insulation-installation-probr1-bo~t_11786~sv_29361",
    #         "end_page": 1039
    #     }, {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-house-framing-probr1-bo~t_11786~sv_29348",
    #         "end_page": 1424
    #     }, {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-garage-building-probr1-bo~t_11786~sv_29308",
    #         "end_page": 1749
    #     }, {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-foundation-repair-probr1-bo~t_11786~sv_29299",
    #         "end_page": 691
    #     }, {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-foundation-construction-probr1-bo~t_11786~sv_29298",
    #         "end_page": 1001
    #     }, {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-fireproofing-probr1-bo~t_11786~sv_29292",
    #         "end_page": 309
    #     }, {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-energy-efficient-homes-probr1-bo~t_11786~sv_29277",
    #         "end_page": 840
    #     }, {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-earthquake-retrofitting-probr1-bo~t_11786~sv_29269",
    #         "end_page": 259
    #     }, {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-drywall-texturing-probr1-bo~t_11786~sv_29267",
    #         "end_page": 1110
    #     }, {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-custom-home-bars-probr1-bo~t_11786~sv_29221",
    #         "end_page": 1054
    #     }, {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-crown-molding-installation-probr1-bo~t_11786~sv_29203",
    #         "end_page": 1468
    #     }, {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-carport-installation-probr1-bo~t_11786~sv_29181",
    #         "end_page": 863
    #     }, {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-soffit-repair-probr1-bo~t_11786~sv_29475",
    #         "end_page": 648
    #     },
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-basement-waterproofing-probr1-bo~t_11786~sv_29154",
    #         "end_page": 627
    #     },
    #     {
    #         "url": "https://www.houzz.com/professionals/general-contractor/project-type-baseboard-installation-probr1-bo~t_11786~sv_29152",
    #         "end_page": 1540
    #     }
    # ]

    BASE_CONFIG = {
        "USE_PROXY": True,
        "CLASH_API_HOST": "http://127.0.0.1:9097",
        "CLASH_API_SECRET": "set-your-secret",  # <-- REPLACE WITH YOUR CLASH SECRET if any
        "CLASH_PROXY_PORT": 7897,
        "CLASH_POLICY_GROUP": "GLOBAL",
        "LATENCY_TEST_TIMEOUT": 5000,
        "START_PAGE": 1,
        "RETRY_COUNT": 2,
        "REQUEST_TIMEOUT": 30,
    }

    session = requests.Session()
    session.headers.update({
                               "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"})

    tasks_queue = deque(initial_tasks)
    clash_controller = None

    if BASE_CONFIG["USE_PROXY"]:
        logging.info("代理模式已启用。正在初始化Clash API控制器...")
        clash_controller = ClashAPI(BASE_CONFIG['CLASH_API_HOST'], BASE_CONFIG['CLASH_API_SECRET'])
        clash_proxy_url = f'http://127.0.0.1:{BASE_CONFIG["CLASH_PROXY_PORT"]}'
        session.proxies = {'http': clash_proxy_url, 'https': clash_proxy_url}
        logging.info(f"Requests Session已配置使用Clash代理: {clash_proxy_url}")
    else:
        logging.info("代理模式已禁用。所有请求将直接发送。")

    try:
        while tasks_queue:
            current_task = tasks_queue.popleft()
            base_url = current_task["url"]
            # 使用正则表达式提取类目名称
            import re
            pattern = r'/professionals/([^/]+)(?:/|$)'
            match = re.search(pattern, base_url)
            if match:
                category_name = match.group(1)
            else:
                # 备用方法
                category_name = base_url.split('/')[4]

            logging.info(
                f"\n{'=' * 25} 准备处理任务: {category_name} (队列中剩余 {len(tasks_queue)} 个任务) {'=' * 25}")

            task_result = 'failure'  # Default task result to failure

            if BASE_CONFIG["USE_PROXY"]:
                # --- Per-Loop Latency Re-Test ---
                sorted_nodes = get_and_sort_nodes_by_latency(clash_controller, BASE_CONFIG['CLASH_POLICY_GROUP'],
                                                             BASE_CONFIG['LATENCY_TEST_TIMEOUT'])

                if not sorted_nodes:
                    logging.error("本次延迟测试未找到可用节点。将任务放回队列并等待60秒。")
                    tasks_queue.append(current_task)
                    time.sleep(60)
                    continue

                current_node = sorted_nodes[0]
                logging.info(f"--- 为任务 '{category_name}' 选择当前最优节点: [{current_node}] ---")

                if clash_controller.switch_node(BASE_CONFIG['CLASH_POLICY_GROUP'], current_node):
                    # Successfully switched node, now run the task
                    task_config = BASE_CONFIG.copy()
                    task_config["END_PAGE"] = current_task["end_page"]
                    task_result = run_collection_task(base_url, task_config, session)
                else:
                    logging.error(f"切换到最优节点 [{current_node}] 失败。此任务将被重新排队。")
                    # task_result remains 'failure'
            else:  # Direct connection mode
                task_config = BASE_CONFIG.copy()
                task_config["END_PAGE"] = current_task["end_page"]
                task_result = run_collection_task(base_url, task_config, session)

            # Evaluate task result
            if task_result == 'success':
                logging.info(f"🎉 任务 '{category_name}' 成功完成！已从队列中移除。")
            else:  # task_result == 'failure'
                logging.warning(f"⚠️ 任务 '{category_name}' 本次执行失败。将放回队列末尾等待下一次重试。")
                tasks_queue.append(current_task)
                time.sleep(10)  # Wait a bit after a failure before starting the next task

        logging.info(f"\n{'=' * 30} 🎉🎉🎉 所有任务均已成功处理完毕! 🎉🎉🎉 {'=' * 30}")

    except KeyboardInterrupt:
        logging.info("\n程序被用户手动中断。")
        sys.exit(0)
    except Exception as e:
        logging.critical(f"!!!!!!!!!!!!!!!!!!!!!!!!!!! 程序因意外的致命错误而终止 !!!!!!!!!!!!!!!!!!!!!!!!!!!")
        logging.critical(f"错误详情: {e}", exc_info=True)
        sys.exit(1)


if __name__ == "__main__":
    main()